{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "redirector stdout\n",
      "OrderedDict([('dataset', 'PETA'),\n",
      "             ('model', 'se_resnet101'),\n",
      "             ('debug', True),\n",
      "             ('batchsize', 64),\n",
      "             ('train_epoch', 50),\n",
      "             ('height', 256),\n",
      "             ('width', 192),\n",
      "             ('lr_ft', 0.01),\n",
      "             ('lr_new', 0.1),\n",
      "             ('classifier', 'base'),\n",
      "             ('momentum', 0.9),\n",
      "             ('weight_decay', 0.0005),\n",
      "             ('train_split', 'trainval'),\n",
      "             ('valid_split', 'test'),\n",
      "             ('device', 0),\n",
      "             ('redirector', True),\n",
      "             ('use_bn', True)])\n",
      "------------------------------------------------------------\n",
      "use GPU0 for training\n",
      "train set: PETA trainval, test set: test\n",
      "Compose(\n",
      "    Resize(size=(256, 192), interpolation=PIL.Image.BILINEAR)\n",
      "    Pad(padding=10, fill=0, padding_mode=constant)\n",
      "    RandomCrop(size=(256, 192), padding=None)\n",
      "    RandomHorizontalFlip(p=0.5)\n",
      "    ToTensor()\n",
      "    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
      ")\n",
      "trainval set: 11400, test set: 7600, attr_num : 116\n",
      "2020-09-27_03-36-51, Step 19/179 in Ep 0, 0.47s  train_loss:69.8020\n",
      "2020-09-27_03-37-24, Step 39/179 in Ep 0, 0.46s  train_loss:45.5845\n",
      "2020-09-27_03-37-57, Step 59/179 in Ep 0, 0.47s  train_loss:40.1992\n",
      "2020-09-27_03-38-30, Step 79/179 in Ep 0, 0.47s  train_loss:36.8112\n",
      "2020-09-27_03-39-02, Step 99/179 in Ep 0, 0.46s  train_loss:32.7357\n",
      "2020-09-27_03-39-34, Step 119/179 in Ep 0, 0.47s  train_loss:31.0475\n",
      "2020-09-27_03-40-07, Step 139/179 in Ep 0, 0.46s  train_loss:30.3486\n",
      "2020-09-27_03-40-40, Step 159/179 in Ep 0, 0.47s  train_loss:27.8484\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_03-41-09, Step 178/179 in Ep 0, 0.17s  train_loss:39.9021\n",
      "Epoch 0, LR 0.1, Train_Time 292.76s, Loss: 42.0236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [02:34<00:00,  1.30s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.7169,  pos_recall: 0.4785 , neg_recall: 0.9554 \n",
      " Acc: 0.6891, Prec: 0.8124, Rec: 0.7895, F1: 0.7983\n",
      "2020-09-27_03-43-44\n",
      "------------------------------------------------------------\n",
      "2020-09-27_03-43-59, Step 19/179 in Ep 1, 0.48s  train_loss:24.8763\n",
      "2020-09-27_03-44-12, Step 39/179 in Ep 1, 0.48s  train_loss:25.0532\n",
      "2020-09-27_03-44-25, Step 59/179 in Ep 1, 0.48s  train_loss:23.6043\n",
      "2020-09-27_03-44-38, Step 79/179 in Ep 1, 0.48s  train_loss:22.6646\n",
      "2020-09-27_03-44-51, Step 99/179 in Ep 1, 0.48s  train_loss:22.8539\n",
      "2020-09-27_03-45-05, Step 119/179 in Ep 1, 0.48s  train_loss:23.5387\n",
      "2020-09-27_03-45-18, Step 139/179 in Ep 1, 0.48s  train_loss:21.2678\n",
      "2020-09-27_03-45-31, Step 159/179 in Ep 1, 0.48s  train_loss:20.0959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_03-45-43, Step 178/179 in Ep 1, 0.17s  train_loss:33.0223\n",
      "Epoch 1, LR 0.1, Train_Time 117.06s, Loss: 23.5264\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.7609,  pos_recall: 0.5590 , neg_recall: 0.9627 \n",
      " Acc: 0.7315, Prec: 0.8384, Rec: 0.8196, F1: 0.8267\n",
      "2020-09-27_03-46-18\n",
      "------------------------------------------------------------\n",
      "2020-09-27_03-46-33, Step 19/179 in Ep 2, 0.46s  train_loss:17.5531\n",
      "2020-09-27_03-46-45, Step 39/179 in Ep 2, 0.46s  train_loss:18.9550\n",
      "2020-09-27_03-46-58, Step 59/179 in Ep 2, 0.46s  train_loss:18.6604\n",
      "2020-09-27_03-47-11, Step 79/179 in Ep 2, 0.47s  train_loss:22.0296\n",
      "2020-09-27_03-47-23, Step 99/179 in Ep 2, 0.47s  train_loss:18.0436\n",
      "2020-09-27_03-47-36, Step 119/179 in Ep 2, 0.47s  train_loss:17.3604\n",
      "2020-09-27_03-47-49, Step 139/179 in Ep 2, 0.46s  train_loss:18.5305\n",
      "2020-09-27_03-48-02, Step 159/179 in Ep 2, 0.47s  train_loss:17.9822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_03-48-13, Step 178/179 in Ep 2, 0.15s  train_loss:25.7698\n",
      "Epoch 2, LR 0.1, Train_Time 113.43s, Loss: 19.1908\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.7806,  pos_recall: 0.5963 , neg_recall: 0.9649 \n",
      " Acc: 0.7548, Prec: 0.8473, Rec: 0.8412, F1: 0.8424\n",
      "2020-09-27_03-48-48\n",
      "------------------------------------------------------------\n",
      "2020-09-27_03-49-03, Step 19/179 in Ep 3, 0.47s  train_loss:18.8632\n",
      "2020-09-27_03-49-16, Step 39/179 in Ep 3, 0.47s  train_loss:14.3364\n",
      "2020-09-27_03-49-28, Step 59/179 in Ep 3, 0.47s  train_loss:15.9487\n",
      "2020-09-27_03-49-41, Step 79/179 in Ep 3, 0.47s  train_loss:16.9950\n",
      "2020-09-27_03-49-54, Step 99/179 in Ep 3, 0.47s  train_loss:16.5545\n",
      "2020-09-27_03-50-06, Step 119/179 in Ep 3, 0.46s  train_loss:16.0769\n",
      "2020-09-27_03-50-19, Step 139/179 in Ep 3, 0.47s  train_loss:14.9257\n",
      "2020-09-27_03-50-31, Step 159/179 in Ep 3, 0.46s  train_loss:16.1409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_03-50-43, Step 178/179 in Ep 3, 0.16s  train_loss:22.6673\n",
      "Epoch 3, LR 0.1, Train_Time 112.60s, Loss: 16.2757\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.7909,  pos_recall: 0.6148 , neg_recall: 0.9670 \n",
      " Acc: 0.7667, Prec: 0.8553, Rec: 0.8499, F1: 0.8509\n",
      "2020-09-27_03-51-18\n",
      "------------------------------------------------------------\n",
      "2020-09-27_03-51-33, Step 19/179 in Ep 4, 0.47s  train_loss:15.1202\n",
      "2020-09-27_03-51-46, Step 39/179 in Ep 4, 0.47s  train_loss:12.9501\n",
      "2020-09-27_03-51-58, Step 59/179 in Ep 4, 0.46s  train_loss:15.5762\n",
      "2020-09-27_03-52-13, Step 79/179 in Ep 4, 0.46s  train_loss:14.3212\n",
      "2020-09-27_03-52-26, Step 99/179 in Ep 4, 0.47s  train_loss:13.1933\n",
      "2020-09-27_03-52-39, Step 119/179 in Ep 4, 0.47s  train_loss:15.7372\n",
      "2020-09-27_03-52-52, Step 139/179 in Ep 4, 0.47s  train_loss:16.3903\n",
      "2020-09-27_03-53-05, Step 159/179 in Ep 4, 0.46s  train_loss:13.7472\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_03-53-17, Step 178/179 in Ep 4, 0.15s  train_loss:23.0242\n",
      "Epoch 4, LR 0.1, Train_Time 116.10s, Loss: 14.1068\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.7943,  pos_recall: 0.6222 , neg_recall: 0.9664 \n",
      " Acc: 0.7731, Prec: 0.8590, Rec: 0.8536, F1: 0.8546\n",
      "2020-09-27_03-53-53\n",
      "------------------------------------------------------------\n",
      "2020-09-27_03-54-08, Step 19/179 in Ep 5, 0.46s  train_loss:11.9440\n",
      "2020-09-27_03-54-21, Step 39/179 in Ep 5, 0.47s  train_loss:11.6462\n",
      "2020-09-27_03-54-34, Step 59/179 in Ep 5, 0.47s  train_loss:11.3192\n",
      "2020-09-27_03-54-47, Step 79/179 in Ep 5, 0.46s  train_loss:11.5802\n",
      "2020-09-27_03-54-59, Step 99/179 in Ep 5, 0.47s  train_loss:12.0528\n",
      "2020-09-27_03-55-12, Step 119/179 in Ep 5, 0.47s  train_loss:14.4781\n",
      "2020-09-27_03-55-25, Step 139/179 in Ep 5, 0.47s  train_loss:12.4845\n",
      "2020-09-27_03-55-38, Step 159/179 in Ep 5, 0.47s  train_loss:12.4381\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_03-55-50, Step 178/179 in Ep 5, 0.16s  train_loss:28.8404\n",
      "Epoch 5, LR 0.1, Train_Time 115.13s, Loss: 12.3043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8033,  pos_recall: 0.6388 , neg_recall: 0.9677 \n",
      " Acc: 0.7781, Prec: 0.8590, Rec: 0.8611, F1: 0.8585\n",
      "2020-09-27_03-56-25\n",
      "------------------------------------------------------------\n",
      "2020-09-27_03-56-39, Step 19/179 in Ep 6, 0.46s  train_loss:10.0831\n",
      "2020-09-27_03-56-53, Step 39/179 in Ep 6, 0.46s  train_loss:9.0880\n",
      "2020-09-27_03-57-06, Step 59/179 in Ep 6, 0.46s  train_loss:10.5984\n",
      "2020-09-27_03-57-19, Step 79/179 in Ep 6, 0.47s  train_loss:9.5179\n",
      "2020-09-27_03-57-32, Step 99/179 in Ep 6, 0.46s  train_loss:12.5364\n",
      "2020-09-27_03-57-44, Step 119/179 in Ep 6, 0.47s  train_loss:9.1952\n",
      "2020-09-27_03-57-57, Step 139/179 in Ep 6, 0.47s  train_loss:11.4261\n",
      "2020-09-27_03-58-10, Step 159/179 in Ep 6, 0.47s  train_loss:9.7651\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_03-58-22, Step 178/179 in Ep 6, 0.15s  train_loss:26.1412\n",
      "Epoch 6, LR 0.1, Train_Time 114.88s, Loss: 10.7573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:34<00:00,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8069,  pos_recall: 0.6439 , neg_recall: 0.9699 \n",
      " Acc: 0.7747, Prec: 0.8618, Rec: 0.8511, F1: 0.8547\n",
      "2020-09-27_03-58-57\n",
      "------------------------------------------------------------\n",
      "2020-09-27_03-59-11, Step 19/179 in Ep 7, 0.47s  train_loss:8.9052\n",
      "2020-09-27_03-59-24, Step 39/179 in Ep 7, 0.46s  train_loss:7.0993\n",
      "2020-09-27_03-59-36, Step 59/179 in Ep 7, 0.47s  train_loss:8.0410\n",
      "2020-09-27_03-59-49, Step 79/179 in Ep 7, 0.46s  train_loss:8.1910\n",
      "2020-09-27_04-00-02, Step 99/179 in Ep 7, 0.47s  train_loss:9.6375\n",
      "2020-09-27_04-00-19, Step 119/179 in Ep 7, 0.48s  train_loss:8.0334\n",
      "2020-09-27_04-00-34, Step 139/179 in Ep 7, 0.48s  train_loss:9.4227\n",
      "2020-09-27_04-00-47, Step 159/179 in Ep 7, 0.48s  train_loss:10.0736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-00-59, Step 178/179 in Ep 7, 0.17s  train_loss:18.8996\n",
      "Epoch 7, LR 0.1, Train_Time 120.43s, Loss: 9.2886\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8039,  pos_recall: 0.6386 , neg_recall: 0.9693 \n",
      " Acc: 0.7852, Prec: 0.8672, Rec: 0.8611, F1: 0.8626\n",
      "2020-09-27_04-01-34\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-01-47, Step 19/179 in Ep 8, 0.46s  train_loss:7.4919\n",
      "2020-09-27_04-01-59, Step 39/179 in Ep 8, 0.46s  train_loss:9.0005\n",
      "2020-09-27_04-02-12, Step 59/179 in Ep 8, 0.46s  train_loss:9.0790\n",
      "2020-09-27_04-02-25, Step 79/179 in Ep 8, 0.47s  train_loss:7.9319\n",
      "2020-09-27_04-02-38, Step 99/179 in Ep 8, 0.47s  train_loss:7.2691\n",
      "2020-09-27_04-02-51, Step 119/179 in Ep 8, 0.46s  train_loss:9.5678\n",
      "2020-09-27_04-03-04, Step 139/179 in Ep 8, 0.47s  train_loss:8.6827\n",
      "2020-09-27_04-03-16, Step 159/179 in Ep 8, 0.46s  train_loss:10.3410\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-03-28, Step 178/179 in Ep 8, 0.16s  train_loss:24.6300\n",
      "Epoch 8, LR 0.1, Train_Time 114.15s, Loss: 8.2079\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.7988,  pos_recall: 0.6300 , neg_recall: 0.9676 \n",
      " Acc: 0.7873, Prec: 0.8654, Rec: 0.8661, F1: 0.8643\n",
      "2020-09-27_04-04-04\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-04-17, Step 19/179 in Ep 9, 0.47s  train_loss:8.0493\n",
      "2020-09-27_04-04-30, Step 39/179 in Ep 9, 0.46s  train_loss:5.7358\n",
      "2020-09-27_04-04-42, Step 59/179 in Ep 9, 0.47s  train_loss:6.2370\n",
      "2020-09-27_04-04-55, Step 79/179 in Ep 9, 0.47s  train_loss:8.1318\n",
      "2020-09-27_04-05-08, Step 99/179 in Ep 9, 0.46s  train_loss:8.0312\n",
      "2020-09-27_04-05-21, Step 119/179 in Ep 9, 0.47s  train_loss:6.7550\n",
      "2020-09-27_04-05-34, Step 139/179 in Ep 9, 0.47s  train_loss:9.6246\n",
      "2020-09-27_04-05-46, Step 159/179 in Ep 9, 0.47s  train_loss:6.9101\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-05-58, Step 178/179 in Ep 9, 0.16s  train_loss:22.1674\n",
      "Epoch 9, LR 0.1, Train_Time 114.19s, Loss: 7.3271\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8101,  pos_recall: 0.6511 , neg_recall: 0.9691 \n",
      " Acc: 0.7877, Prec: 0.8676, Rec: 0.8639, F1: 0.8643\n",
      "2020-09-27_04-06-34\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-06-49, Step 19/179 in Ep 10, 0.46s  train_loss:6.1633\n",
      "2020-09-27_04-07-01, Step 39/179 in Ep 10, 0.47s  train_loss:5.7032\n",
      "2020-09-27_04-07-14, Step 59/179 in Ep 10, 0.47s  train_loss:7.4696\n",
      "2020-09-27_04-07-27, Step 79/179 in Ep 10, 0.46s  train_loss:6.0793\n",
      "2020-09-27_04-07-40, Step 99/179 in Ep 10, 0.47s  train_loss:6.9474\n",
      "2020-09-27_04-07-53, Step 119/179 in Ep 10, 0.46s  train_loss:6.5709\n",
      "2020-09-27_04-08-05, Step 139/179 in Ep 10, 0.47s  train_loss:6.5455\n",
      "2020-09-27_04-08-18, Step 159/179 in Ep 10, 0.47s  train_loss:6.8147\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-08-30, Step 178/179 in Ep 10, 0.16s  train_loss:15.8534\n",
      "Epoch 10, LR 0.1, Train_Time 114.31s, Loss: 6.5874\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8070,  pos_recall: 0.6431 , neg_recall: 0.9708 \n",
      " Acc: 0.7895, Prec: 0.8737, Rec: 0.8605, F1: 0.8656\n",
      "2020-09-27_04-09-05\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-09-18, Step 19/179 in Ep 11, 0.47s  train_loss:4.9913\n",
      "2020-09-27_04-09-31, Step 39/179 in Ep 11, 0.47s  train_loss:5.0776\n",
      "2020-09-27_04-09-44, Step 59/179 in Ep 11, 0.46s  train_loss:4.7096\n",
      "2020-09-27_04-09-57, Step 79/179 in Ep 11, 0.47s  train_loss:4.5940\n",
      "2020-09-27_04-10-10, Step 99/179 in Ep 11, 0.47s  train_loss:4.6392\n",
      "2020-09-27_04-10-23, Step 119/179 in Ep 11, 0.47s  train_loss:3.8484\n",
      "2020-09-27_04-10-36, Step 139/179 in Ep 11, 0.47s  train_loss:4.7078\n",
      "2020-09-27_04-10-49, Step 159/179 in Ep 11, 0.46s  train_loss:4.2307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-11-01, Step 178/179 in Ep 11, 0.15s  train_loss:18.0072\n",
      "Epoch 11, LR 0.010000000000000002, Train_Time 115.20s, Loss: 4.3926\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:34<00:00,  3.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8164,  pos_recall: 0.6590 , neg_recall: 0.9737 \n",
      " Acc: 0.8103, Prec: 0.8892, Rec: 0.8743, F1: 0.8803\n",
      "2020-09-27_04-11-35\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-11-50, Step 19/179 in Ep 12, 0.47s  train_loss:3.2608\n",
      "2020-09-27_04-12-03, Step 39/179 in Ep 12, 0.46s  train_loss:3.5039\n",
      "2020-09-27_04-12-15, Step 59/179 in Ep 12, 0.47s  train_loss:3.2567\n",
      "2020-09-27_04-12-28, Step 79/179 in Ep 12, 0.47s  train_loss:3.2346\n",
      "2020-09-27_04-12-41, Step 99/179 in Ep 12, 0.47s  train_loss:4.3904\n",
      "2020-09-27_04-12-53, Step 119/179 in Ep 12, 0.47s  train_loss:3.8791\n",
      "2020-09-27_04-13-06, Step 139/179 in Ep 12, 0.47s  train_loss:2.8255\n",
      "2020-09-27_04-13-19, Step 159/179 in Ep 12, 0.46s  train_loss:2.9741\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-13-30, Step 178/179 in Ep 12, 0.16s  train_loss:25.0648\n",
      "Epoch 12, LR 0.010000000000000002, Train_Time 113.04s, Loss: 3.4206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8140,  pos_recall: 0.6546 , neg_recall: 0.9734 \n",
      " Acc: 0.8122, Prec: 0.8912, Rec: 0.8748, F1: 0.8816\n",
      "2020-09-27_04-14-06\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-14-19, Step 19/179 in Ep 13, 0.46s  train_loss:3.0496\n",
      "2020-09-27_04-14-31, Step 39/179 in Ep 13, 0.46s  train_loss:2.6376\n",
      "2020-09-27_04-14-44, Step 59/179 in Ep 13, 0.46s  train_loss:2.8047\n",
      "2020-09-27_04-14-56, Step 79/179 in Ep 13, 0.46s  train_loss:2.4227\n",
      "2020-09-27_04-15-09, Step 99/179 in Ep 13, 0.46s  train_loss:2.7529\n",
      "2020-09-27_04-15-22, Step 119/179 in Ep 13, 0.47s  train_loss:3.4285\n",
      "2020-09-27_04-15-35, Step 139/179 in Ep 13, 0.47s  train_loss:3.1170\n",
      "2020-09-27_04-15-47, Step 159/179 in Ep 13, 0.47s  train_loss:2.8708\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-15-59, Step 178/179 in Ep 13, 0.16s  train_loss:21.2458\n",
      "Epoch 13, LR 0.010000000000000002, Train_Time 113.47s, Loss: 2.9152\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [01:03<00:00,  1.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8190,  pos_recall: 0.6642 , neg_recall: 0.9739 \n",
      " Acc: 0.8125, Prec: 0.8918, Rec: 0.8746, F1: 0.8818\n",
      "2020-09-27_04-17-03\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-17-19, Step 19/179 in Ep 14, 0.47s  train_loss:2.5999\n",
      "2020-09-27_04-17-34, Step 39/179 in Ep 14, 0.47s  train_loss:2.5960\n",
      "2020-09-27_04-17-53, Step 59/179 in Ep 14, 0.47s  train_loss:2.7397\n",
      "2020-09-27_04-18-13, Step 79/179 in Ep 14, 0.47s  train_loss:2.2299\n",
      "2020-09-27_04-18-30, Step 99/179 in Ep 14, 0.47s  train_loss:2.1155\n",
      "2020-09-27_04-18-47, Step 119/179 in Ep 14, 0.46s  train_loss:2.3932\n",
      "2020-09-27_04-19-06, Step 139/179 in Ep 14, 0.47s  train_loss:2.7807\n",
      "2020-09-27_04-19-25, Step 159/179 in Ep 14, 0.47s  train_loss:2.1644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-19-41, Step 178/179 in Ep 14, 0.16s  train_loss:17.3070\n",
      "Epoch 14, LR 0.010000000000000002, Train_Time 155.19s, Loss: 2.5874\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [01:10<00:00,  1.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8171,  pos_recall: 0.6601 , neg_recall: 0.9740 \n",
      " Acc: 0.8131, Prec: 0.8922, Rec: 0.8750, F1: 0.8822\n",
      "2020-09-27_04-20-51\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-21-10, Step 19/179 in Ep 15, 0.46s  train_loss:2.3711\n",
      "2020-09-27_04-21-26, Step 39/179 in Ep 15, 0.47s  train_loss:2.2333\n",
      "2020-09-27_04-21-44, Step 59/179 in Ep 15, 0.47s  train_loss:1.8698\n",
      "2020-09-27_04-22-00, Step 79/179 in Ep 15, 0.47s  train_loss:2.2199\n",
      "2020-09-27_04-22-20, Step 99/179 in Ep 15, 0.46s  train_loss:2.4329\n",
      "2020-09-27_04-22-38, Step 119/179 in Ep 15, 0.46s  train_loss:1.8553\n",
      "2020-09-27_04-22-56, Step 139/179 in Ep 15, 0.47s  train_loss:1.8533\n",
      "2020-09-27_04-23-13, Step 159/179 in Ep 15, 0.46s  train_loss:2.2132\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-23-29, Step 178/179 in Ep 15, 0.15s  train_loss:14.6606\n",
      "Epoch 15, LR 0.010000000000000002, Train_Time 157.50s, Loss: 2.3047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [01:08<00:00,  1.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8175,  pos_recall: 0.6608 , neg_recall: 0.9743 \n",
      " Acc: 0.8137, Prec: 0.8934, Rec: 0.8747, F1: 0.8826\n",
      "2020-09-27_04-24-38\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-24-57, Step 19/179 in Ep 16, 0.47s  train_loss:1.8978\n",
      "2020-09-27_04-25-17, Step 39/179 in Ep 16, 0.47s  train_loss:2.1294\n",
      "2020-09-27_04-25-41, Step 59/179 in Ep 16, 0.46s  train_loss:2.3807\n",
      "2020-09-27_04-26-07, Step 79/179 in Ep 16, 0.46s  train_loss:1.9294\n",
      "2020-09-27_04-26-27, Step 99/179 in Ep 16, 0.46s  train_loss:1.9634\n",
      "2020-09-27_04-26-44, Step 119/179 in Ep 16, 0.47s  train_loss:2.2862\n",
      "2020-09-27_04-27-00, Step 139/179 in Ep 16, 0.46s  train_loss:1.9029\n",
      "2020-09-27_04-27-18, Step 159/179 in Ep 16, 0.46s  train_loss:1.9689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-27-35, Step 178/179 in Ep 16, 0.16s  train_loss:9.6917\n",
      "Epoch 16, LR 0.010000000000000002, Train_Time 177.51s, Loss: 2.0525\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:40<00:00,  2.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8182,  pos_recall: 0.6624 , neg_recall: 0.9741 \n",
      " Acc: 0.8139, Prec: 0.8940, Rec: 0.8744, F1: 0.8827\n",
      "2020-09-27_04-28-16\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-28-29, Step 19/179 in Ep 17, 0.46s  train_loss:2.2207\n",
      "2020-09-27_04-28-42, Step 39/179 in Ep 17, 0.47s  train_loss:1.9754\n",
      "2020-09-27_04-28-55, Step 59/179 in Ep 17, 0.47s  train_loss:2.1992\n",
      "2020-09-27_04-29-08, Step 79/179 in Ep 17, 0.46s  train_loss:1.7958\n",
      "2020-09-27_04-29-20, Step 99/179 in Ep 17, 0.47s  train_loss:1.7082\n",
      "2020-09-27_04-29-33, Step 119/179 in Ep 17, 0.46s  train_loss:2.0155\n",
      "2020-09-27_04-29-46, Step 139/179 in Ep 17, 0.46s  train_loss:1.5799\n",
      "2020-09-27_04-29-58, Step 159/179 in Ep 17, 0.47s  train_loss:2.1800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-30-10, Step 178/179 in Ep 17, 0.16s  train_loss:15.3625\n",
      "Epoch 17, LR 0.0010000000000000002, Train_Time 113.96s, Loss: 1.9523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8205,  pos_recall: 0.6666 , neg_recall: 0.9744 \n",
      " Acc: 0.8146, Prec: 0.8937, Rec: 0.8753, F1: 0.8831\n",
      "2020-09-27_04-30-46\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-31-00, Step 19/179 in Ep 18, 0.47s  train_loss:1.6873\n",
      "2020-09-27_04-31-13, Step 39/179 in Ep 18, 0.47s  train_loss:1.8545\n",
      "2020-09-27_04-31-26, Step 59/179 in Ep 18, 0.47s  train_loss:1.7687\n",
      "2020-09-27_04-31-39, Step 79/179 in Ep 18, 0.47s  train_loss:1.4537\n",
      "2020-09-27_04-31-52, Step 99/179 in Ep 18, 0.48s  train_loss:1.9378\n",
      "2020-09-27_04-32-06, Step 119/179 in Ep 18, 0.47s  train_loss:1.5730\n",
      "2020-09-27_04-32-19, Step 139/179 in Ep 18, 0.47s  train_loss:1.5560\n",
      "2020-09-27_04-32-32, Step 159/179 in Ep 18, 0.48s  train_loss:2.1561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-32-44, Step 178/179 in Ep 18, 0.17s  train_loss:7.6438\n",
      "Epoch 18, LR 0.0010000000000000002, Train_Time 116.46s, Loss: 1.8224\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8180,  pos_recall: 0.6614 , neg_recall: 0.9747 \n",
      " Acc: 0.8144, Prec: 0.8952, Rec: 0.8740, F1: 0.8830\n",
      "2020-09-27_04-33-19\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-33-32, Step 19/179 in Ep 19, 0.47s  train_loss:1.7055\n",
      "2020-09-27_04-33-45, Step 39/179 in Ep 19, 0.46s  train_loss:1.7678\n",
      "2020-09-27_04-33-58, Step 59/179 in Ep 19, 0.47s  train_loss:1.9890\n",
      "2020-09-27_04-34-10, Step 79/179 in Ep 19, 0.46s  train_loss:1.9662\n",
      "2020-09-27_04-34-23, Step 99/179 in Ep 19, 0.46s  train_loss:1.5365\n",
      "2020-09-27_04-34-35, Step 119/179 in Ep 19, 0.46s  train_loss:1.5528\n",
      "2020-09-27_04-34-48, Step 139/179 in Ep 19, 0.47s  train_loss:1.9635\n",
      "2020-09-27_04-35-01, Step 159/179 in Ep 19, 0.46s  train_loss:1.6723\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-35-13, Step 178/179 in Ep 19, 0.16s  train_loss:14.8443\n",
      "Epoch 19, LR 0.0010000000000000002, Train_Time 113.62s, Loss: 1.8389\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8209,  pos_recall: 0.6672 , neg_recall: 0.9746 \n",
      " Acc: 0.8152, Prec: 0.8947, Rec: 0.8752, F1: 0.8835\n",
      "2020-09-27_04-35-49\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-36-04, Step 19/179 in Ep 20, 0.46s  train_loss:1.7136\n",
      "2020-09-27_04-36-16, Step 39/179 in Ep 20, 0.46s  train_loss:1.4615\n",
      "2020-09-27_04-36-29, Step 59/179 in Ep 20, 0.46s  train_loss:1.7051\n",
      "2020-09-27_04-36-42, Step 79/179 in Ep 20, 0.46s  train_loss:1.5412\n",
      "2020-09-27_04-36-55, Step 99/179 in Ep 20, 0.46s  train_loss:1.8415\n",
      "2020-09-27_04-37-07, Step 119/179 in Ep 20, 0.46s  train_loss:2.1067\n",
      "2020-09-27_04-37-20, Step 139/179 in Ep 20, 0.46s  train_loss:1.7410\n",
      "2020-09-27_04-37-33, Step 159/179 in Ep 20, 0.45s  train_loss:1.6561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-37-44, Step 178/179 in Ep 20, 0.15s  train_loss:10.6258\n",
      "Epoch 20, LR 0.0010000000000000002, Train_Time 113.40s, Loss: 1.7991\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8181,  pos_recall: 0.6620 , neg_recall: 0.9741 \n",
      " Acc: 0.8152, Prec: 0.8942, Rec: 0.8759, F1: 0.8836\n",
      "2020-09-27_04-38-20\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-38-33, Step 19/179 in Ep 21, 0.46s  train_loss:1.8495\n",
      "2020-09-27_04-38-45, Step 39/179 in Ep 21, 0.46s  train_loss:1.8474\n",
      "2020-09-27_04-38-58, Step 59/179 in Ep 21, 0.47s  train_loss:1.9622\n",
      "2020-09-27_04-39-10, Step 79/179 in Ep 21, 0.45s  train_loss:1.5503\n",
      "2020-09-27_04-39-24, Step 99/179 in Ep 21, 0.46s  train_loss:1.5056\n",
      "2020-09-27_04-39-36, Step 119/179 in Ep 21, 0.45s  train_loss:1.5803\n",
      "2020-09-27_04-39-49, Step 139/179 in Ep 21, 0.47s  train_loss:1.9468\n",
      "2020-09-27_04-40-01, Step 159/179 in Ep 21, 0.46s  train_loss:1.8033\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-40-13, Step 178/179 in Ep 21, 0.16s  train_loss:44.9182\n",
      "Epoch 21, LR 0.0010000000000000002, Train_Time 112.77s, Loss: 1.9833\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:36<00:00,  3.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8189,  pos_recall: 0.6635 , neg_recall: 0.9743 \n",
      " Acc: 0.8141, Prec: 0.8941, Rec: 0.8744, F1: 0.8828\n",
      "2020-09-27_04-40-49\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-41-02, Step 19/179 in Ep 22, 0.46s  train_loss:1.4948\n",
      "2020-09-27_04-41-15, Step 39/179 in Ep 22, 0.46s  train_loss:1.6973\n",
      "2020-09-27_04-41-28, Step 59/179 in Ep 22, 0.47s  train_loss:1.7614\n",
      "2020-09-27_04-41-41, Step 79/179 in Ep 22, 0.47s  train_loss:1.6240\n",
      "2020-09-27_04-41-53, Step 99/179 in Ep 22, 0.46s  train_loss:1.8158\n",
      "2020-09-27_04-42-06, Step 119/179 in Ep 22, 0.46s  train_loss:1.6671\n",
      "2020-09-27_04-42-19, Step 139/179 in Ep 22, 0.46s  train_loss:1.8290\n",
      "2020-09-27_04-42-31, Step 159/179 in Ep 22, 0.47s  train_loss:2.1432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-42-43, Step 178/179 in Ep 22, 0.16s  train_loss:10.2442\n",
      "Epoch 22, LR 0.00010000000000000003, Train_Time 113.50s, Loss: 1.7429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8189,  pos_recall: 0.6634 , neg_recall: 0.9744 \n",
      " Acc: 0.8153, Prec: 0.8944, Rec: 0.8758, F1: 0.8836\n",
      "2020-09-27_04-43-19\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-43-31, Step 19/179 in Ep 23, 0.46s  train_loss:1.6281\n",
      "2020-09-27_04-43-44, Step 39/179 in Ep 23, 0.46s  train_loss:1.4636\n",
      "2020-09-27_04-43-57, Step 59/179 in Ep 23, 0.47s  train_loss:1.6697\n",
      "2020-09-27_04-44-10, Step 79/179 in Ep 23, 0.47s  train_loss:1.4953\n",
      "2020-09-27_04-44-23, Step 99/179 in Ep 23, 0.47s  train_loss:1.6128\n",
      "2020-09-27_04-44-36, Step 119/179 in Ep 23, 0.48s  train_loss:1.6151\n",
      "2020-09-27_04-44-49, Step 139/179 in Ep 23, 0.47s  train_loss:1.5043\n",
      "2020-09-27_04-45-02, Step 159/179 in Ep 23, 0.48s  train_loss:2.0899\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-45-15, Step 178/179 in Ep 23, 0.17s  train_loss:12.4412\n",
      "Epoch 23, LR 0.00010000000000000003, Train_Time 115.73s, Loss: 1.7565\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8206,  pos_recall: 0.6662 , neg_recall: 0.9749 \n",
      " Acc: 0.8141, Prec: 0.8941, Rec: 0.8744, F1: 0.8828\n",
      "2020-09-27_04-45-50\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-46-02, Step 19/179 in Ep 24, 0.46s  train_loss:1.5799\n",
      "2020-09-27_04-46-15, Step 39/179 in Ep 24, 0.47s  train_loss:1.5733\n",
      "2020-09-27_04-46-29, Step 59/179 in Ep 24, 0.46s  train_loss:1.7725\n",
      "2020-09-27_04-46-41, Step 79/179 in Ep 24, 0.46s  train_loss:1.5037\n",
      "2020-09-27_04-46-54, Step 99/179 in Ep 24, 0.46s  train_loss:1.5368\n",
      "2020-09-27_04-47-06, Step 119/179 in Ep 24, 0.47s  train_loss:1.9417\n",
      "2020-09-27_04-47-19, Step 139/179 in Ep 24, 0.46s  train_loss:1.7157\n",
      "2020-09-27_04-47-32, Step 159/179 in Ep 24, 0.45s  train_loss:1.5552\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-47-43, Step 178/179 in Ep 24, 0.16s  train_loss:18.3133\n",
      "Epoch 24, LR 0.00010000000000000003, Train_Time 113.25s, Loss: 1.7783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8196,  pos_recall: 0.6649 , neg_recall: 0.9743 \n",
      " Acc: 0.8145, Prec: 0.8940, Rec: 0.8752, F1: 0.8831\n",
      "2020-09-27_04-48-19\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-48-31, Step 19/179 in Ep 25, 0.47s  train_loss:1.6199\n",
      "2020-09-27_04-48-44, Step 39/179 in Ep 25, 0.45s  train_loss:1.3884\n",
      "2020-09-27_04-48-57, Step 59/179 in Ep 25, 0.46s  train_loss:1.6531\n",
      "2020-09-27_04-49-09, Step 79/179 in Ep 25, 0.46s  train_loss:1.6286\n",
      "2020-09-27_04-49-22, Step 99/179 in Ep 25, 0.46s  train_loss:1.6177\n",
      "2020-09-27_04-49-34, Step 119/179 in Ep 25, 0.46s  train_loss:1.9122\n",
      "2020-09-27_04-49-47, Step 139/179 in Ep 25, 0.46s  train_loss:1.5448\n",
      "2020-09-27_04-50-00, Step 159/179 in Ep 25, 0.47s  train_loss:1.8737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-50-12, Step 178/179 in Ep 25, 0.16s  train_loss:14.6642\n",
      "Epoch 25, LR 0.00010000000000000003, Train_Time 112.80s, Loss: 1.7624\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:34<00:00,  3.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8184,  pos_recall: 0.6626 , neg_recall: 0.9742 \n",
      " Acc: 0.8155, Prec: 0.8934, Rec: 0.8769, F1: 0.8837\n",
      "2020-09-27_04-50-46\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-50-59, Step 19/179 in Ep 26, 0.46s  train_loss:1.5628\n",
      "2020-09-27_04-51-12, Step 39/179 in Ep 26, 0.46s  train_loss:1.7051\n",
      "2020-09-27_04-51-25, Step 59/179 in Ep 26, 0.45s  train_loss:1.6935\n",
      "2020-09-27_04-51-37, Step 79/179 in Ep 26, 0.47s  train_loss:1.6841\n",
      "2020-09-27_04-51-50, Step 99/179 in Ep 26, 0.47s  train_loss:1.6665\n",
      "2020-09-27_04-52-03, Step 119/179 in Ep 26, 0.46s  train_loss:1.5960\n",
      "2020-09-27_04-52-15, Step 139/179 in Ep 26, 0.47s  train_loss:1.8700\n",
      "2020-09-27_04-52-28, Step 159/179 in Ep 26, 0.46s  train_loss:2.0852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-52-40, Step 178/179 in Ep 26, 0.15s  train_loss:12.9451\n",
      "Epoch 26, LR 0.00010000000000000003, Train_Time 113.09s, Loss: 1.7652\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8196,  pos_recall: 0.6651 , neg_recall: 0.9741 \n",
      " Acc: 0.8156, Prec: 0.8936, Rec: 0.8770, F1: 0.8839\n",
      "2020-09-27_04-53-15\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-53-27, Step 19/179 in Ep 27, 0.46s  train_loss:1.4627\n",
      "2020-09-27_04-53-40, Step 39/179 in Ep 27, 0.45s  train_loss:1.5377\n",
      "2020-09-27_04-53-53, Step 59/179 in Ep 27, 0.47s  train_loss:2.0897\n",
      "2020-09-27_04-54-06, Step 79/179 in Ep 27, 0.46s  train_loss:1.5534\n",
      "2020-09-27_04-54-18, Step 99/179 in Ep 27, 0.47s  train_loss:1.8301\n",
      "2020-09-27_04-54-31, Step 119/179 in Ep 27, 0.46s  train_loss:1.6719\n",
      "2020-09-27_04-54-44, Step 139/179 in Ep 27, 0.47s  train_loss:2.0659\n",
      "2020-09-27_04-54-57, Step 159/179 in Ep 27, 0.46s  train_loss:1.5917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-55-09, Step 178/179 in Ep 27, 0.16s  train_loss:13.9105\n",
      "Epoch 27, LR 1.0000000000000004e-05, Train_Time 113.68s, Loss: 1.7730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:36<00:00,  3.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8170,  pos_recall: 0.6599 , neg_recall: 0.9742 \n",
      " Acc: 0.8148, Prec: 0.8946, Rec: 0.8751, F1: 0.8834\n",
      "2020-09-27_04-55-46\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-55-58, Step 19/179 in Ep 28, 0.47s  train_loss:1.5975\n",
      "2020-09-27_04-56-12, Step 39/179 in Ep 28, 0.47s  train_loss:1.6650\n",
      "2020-09-27_04-56-25, Step 59/179 in Ep 28, 0.47s  train_loss:1.6231\n",
      "2020-09-27_04-56-38, Step 79/179 in Ep 28, 0.46s  train_loss:1.7948\n",
      "2020-09-27_04-56-50, Step 99/179 in Ep 28, 0.48s  train_loss:1.7942\n",
      "2020-09-27_04-57-04, Step 119/179 in Ep 28, 0.47s  train_loss:1.5065\n",
      "2020-09-27_04-57-17, Step 139/179 in Ep 28, 0.47s  train_loss:1.6073\n",
      "2020-09-27_04-57-30, Step 159/179 in Ep 28, 0.48s  train_loss:1.7924\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_04-57-42, Step 178/179 in Ep 28, 0.17s  train_loss:15.1532\n",
      "Epoch 28, LR 1.0000000000000004e-05, Train_Time 116.75s, Loss: 1.7619\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:36<00:00,  3.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8179,  pos_recall: 0.6617 , neg_recall: 0.9742 \n",
      " Acc: 0.8157, Prec: 0.8943, Rec: 0.8764, F1: 0.8838\n",
      "2020-09-27_04-58-19\n",
      "------------------------------------------------------------\n",
      "2020-09-27_04-58-32, Step 19/179 in Ep 29, 0.48s  train_loss:2.1307\n",
      "2020-09-27_04-58-45, Step 39/179 in Ep 29, 0.48s  train_loss:1.6885\n",
      "2020-09-27_04-58-58, Step 59/179 in Ep 29, 0.47s  train_loss:1.8318\n",
      "2020-09-27_04-59-12, Step 79/179 in Ep 29, 0.46s  train_loss:1.7676\n",
      "2020-09-27_04-59-25, Step 99/179 in Ep 29, 0.45s  train_loss:1.4767\n",
      "2020-09-27_04-59-38, Step 119/179 in Ep 29, 0.47s  train_loss:1.8294\n",
      "2020-09-27_04-59-51, Step 139/179 in Ep 29, 0.46s  train_loss:1.5053\n",
      "2020-09-27_05-00-03, Step 159/179 in Ep 29, 0.46s  train_loss:1.6516\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-00-15, Step 178/179 in Ep 29, 0.15s  train_loss:13.5129\n",
      "Epoch 29, LR 1.0000000000000004e-05, Train_Time 115.87s, Loss: 1.7407\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8185,  pos_recall: 0.6626 , neg_recall: 0.9744 \n",
      " Acc: 0.8143, Prec: 0.8949, Rec: 0.8740, F1: 0.8829\n",
      "2020-09-27_05-00-51\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-01-04, Step 19/179 in Ep 30, 0.45s  train_loss:1.5520\n",
      "2020-09-27_05-01-16, Step 39/179 in Ep 30, 0.46s  train_loss:1.5618\n",
      "2020-09-27_05-01-29, Step 59/179 in Ep 30, 0.46s  train_loss:1.7139\n",
      "2020-09-27_05-01-41, Step 79/179 in Ep 30, 0.46s  train_loss:1.7301\n",
      "2020-09-27_05-01-54, Step 99/179 in Ep 30, 0.46s  train_loss:1.6082\n",
      "2020-09-27_05-02-07, Step 119/179 in Ep 30, 0.46s  train_loss:1.6815\n",
      "2020-09-27_05-02-20, Step 139/179 in Ep 30, 0.47s  train_loss:1.7674\n",
      "2020-09-27_05-02-32, Step 159/179 in Ep 30, 0.45s  train_loss:1.5273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-02-44, Step 178/179 in Ep 30, 0.16s  train_loss:15.4031\n",
      "Epoch 30, LR 1.0000000000000004e-05, Train_Time 112.91s, Loss: 1.7853\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8206,  pos_recall: 0.6662 , neg_recall: 0.9750 \n",
      " Acc: 0.8138, Prec: 0.8948, Rec: 0.8734, F1: 0.8826\n",
      "2020-09-27_05-03-20\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-03-32, Step 19/179 in Ep 31, 0.47s  train_loss:1.6916\n",
      "2020-09-27_05-03-45, Step 39/179 in Ep 31, 0.47s  train_loss:1.5268\n",
      "2020-09-27_05-03-59, Step 59/179 in Ep 31, 0.48s  train_loss:1.8179\n",
      "2020-09-27_05-04-12, Step 79/179 in Ep 31, 0.47s  train_loss:1.7788\n",
      "2020-09-27_05-04-25, Step 99/179 in Ep 31, 0.48s  train_loss:1.6704\n",
      "2020-09-27_05-04-38, Step 119/179 in Ep 31, 0.47s  train_loss:1.6865\n",
      "2020-09-27_05-04-51, Step 139/179 in Ep 31, 0.47s  train_loss:1.4823\n",
      "2020-09-27_05-05-04, Step 159/179 in Ep 31, 0.47s  train_loss:1.5556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-05-16, Step 178/179 in Ep 31, 0.17s  train_loss:18.9675\n",
      "Epoch 31, LR 1.0000000000000004e-05, Train_Time 116.61s, Loss: 1.8024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8206,  pos_recall: 0.6668 , neg_recall: 0.9744 \n",
      " Acc: 0.8160, Prec: 0.8944, Rec: 0.8768, F1: 0.8842\n",
      "2020-09-27_05-05-52\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-06-04, Step 19/179 in Ep 32, 0.47s  train_loss:1.6674\n",
      "2020-09-27_05-06-17, Step 39/179 in Ep 32, 0.47s  train_loss:1.6085\n",
      "2020-09-27_05-06-30, Step 59/179 in Ep 32, 0.47s  train_loss:1.9075\n",
      "2020-09-27_05-06-42, Step 79/179 in Ep 32, 0.47s  train_loss:1.8857\n",
      "2020-09-27_05-06-55, Step 99/179 in Ep 32, 0.47s  train_loss:1.9050\n",
      "2020-09-27_05-07-08, Step 119/179 in Ep 32, 0.47s  train_loss:1.5823\n",
      "2020-09-27_05-07-21, Step 139/179 in Ep 32, 0.47s  train_loss:1.7291\n",
      "2020-09-27_05-07-34, Step 159/179 in Ep 32, 0.47s  train_loss:2.2395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-07-45, Step 178/179 in Ep 32, 0.15s  train_loss:19.6088\n",
      "Epoch 32, LR 1.0000000000000004e-06, Train_Time 113.50s, Loss: 1.7949\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8175,  pos_recall: 0.6603 , neg_recall: 0.9746 \n",
      " Acc: 0.8144, Prec: 0.8946, Rec: 0.8745, F1: 0.8830\n",
      "2020-09-27_05-08-21\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-08-34, Step 19/179 in Ep 33, 0.48s  train_loss:1.8271\n",
      "2020-09-27_05-08-47, Step 39/179 in Ep 33, 0.46s  train_loss:1.4967\n",
      "2020-09-27_05-09-00, Step 59/179 in Ep 33, 0.47s  train_loss:1.5868\n",
      "2020-09-27_05-09-13, Step 79/179 in Ep 33, 0.47s  train_loss:1.5015\n",
      "2020-09-27_05-09-26, Step 99/179 in Ep 33, 0.47s  train_loss:1.7069\n",
      "2020-09-27_05-09-40, Step 119/179 in Ep 33, 0.48s  train_loss:1.6813\n",
      "2020-09-27_05-09-54, Step 139/179 in Ep 33, 0.47s  train_loss:1.8485\n",
      "2020-09-27_05-10-08, Step 159/179 in Ep 33, 0.48s  train_loss:1.7673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-10-20, Step 178/179 in Ep 33, 0.17s  train_loss:18.6147\n",
      "Epoch 33, LR 1.0000000000000004e-06, Train_Time 119.46s, Loss: 1.7895\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:36<00:00,  3.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8180,  pos_recall: 0.6616 , neg_recall: 0.9745 \n",
      " Acc: 0.8156, Prec: 0.8950, Rec: 0.8757, F1: 0.8839\n",
      "2020-09-27_05-10-57\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-11-10, Step 19/179 in Ep 34, 0.47s  train_loss:1.6040\n",
      "2020-09-27_05-11-23, Step 39/179 in Ep 34, 0.48s  train_loss:2.0312\n",
      "2020-09-27_05-11-36, Step 59/179 in Ep 34, 0.46s  train_loss:1.5333\n",
      "2020-09-27_05-11-50, Step 79/179 in Ep 34, 0.48s  train_loss:1.9868\n",
      "2020-09-27_05-12-03, Step 99/179 in Ep 34, 0.48s  train_loss:1.7250\n",
      "2020-09-27_05-12-16, Step 119/179 in Ep 34, 0.48s  train_loss:1.6409\n",
      "2020-09-27_05-12-29, Step 139/179 in Ep 34, 0.47s  train_loss:1.6356\n",
      "2020-09-27_05-12-42, Step 159/179 in Ep 34, 0.48s  train_loss:2.0016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-12-54, Step 178/179 in Ep 34, 0.17s  train_loss:12.7901\n",
      "Epoch 34, LR 1.0000000000000004e-06, Train_Time 116.93s, Loss: 1.7649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8193,  pos_recall: 0.6641 , neg_recall: 0.9744 \n",
      " Acc: 0.8137, Prec: 0.8937, Rec: 0.8743, F1: 0.8825\n",
      "2020-09-27_05-13-29\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-13-42, Step 19/179 in Ep 35, 0.47s  train_loss:1.7158\n",
      "2020-09-27_05-13-54, Step 39/179 in Ep 35, 0.47s  train_loss:1.7225\n",
      "2020-09-27_05-14-07, Step 59/179 in Ep 35, 0.47s  train_loss:1.8344\n",
      "2020-09-27_05-14-20, Step 79/179 in Ep 35, 0.47s  train_loss:1.7479\n",
      "2020-09-27_05-14-33, Step 99/179 in Ep 35, 0.46s  train_loss:1.8284\n",
      "2020-09-27_05-14-45, Step 119/179 in Ep 35, 0.47s  train_loss:1.6183\n",
      "2020-09-27_05-14-58, Step 139/179 in Ep 35, 0.45s  train_loss:1.7032\n",
      "2020-09-27_05-15-11, Step 159/179 in Ep 35, 0.46s  train_loss:1.6581\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-15-23, Step 178/179 in Ep 35, 0.16s  train_loss:18.6036\n",
      "Epoch 35, LR 1.0000000000000004e-06, Train_Time 113.43s, Loss: 1.7852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8180,  pos_recall: 0.6617 , neg_recall: 0.9744 \n",
      " Acc: 0.8151, Prec: 0.8960, Rec: 0.8740, F1: 0.8835\n",
      "2020-09-27_05-15-58\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-16-11, Step 19/179 in Ep 36, 0.46s  train_loss:1.4575\n",
      "2020-09-27_05-16-29, Step 39/179 in Ep 36, 0.46s  train_loss:2.0435\n",
      "2020-09-27_05-16-50, Step 59/179 in Ep 36, 0.46s  train_loss:1.3918\n",
      "2020-09-27_05-17-18, Step 79/179 in Ep 36, 0.47s  train_loss:1.5848\n",
      "2020-09-27_05-17-34, Step 99/179 in Ep 36, 0.46s  train_loss:1.5894\n",
      "2020-09-27_05-17-47, Step 119/179 in Ep 36, 0.46s  train_loss:1.4102\n",
      "2020-09-27_05-18-03, Step 139/179 in Ep 36, 0.47s  train_loss:1.9133\n",
      "2020-09-27_05-18-22, Step 159/179 in Ep 36, 0.46s  train_loss:1.7790\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-18-40, Step 178/179 in Ep 36, 0.18s  train_loss:11.6447\n",
      "Epoch 36, LR 1.0000000000000004e-06, Train_Time 162.56s, Loss: 1.7272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [01:15<00:00,  1.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8186,  pos_recall: 0.6630 , neg_recall: 0.9742 \n",
      " Acc: 0.8153, Prec: 0.8945, Rec: 0.8757, F1: 0.8837\n",
      "2020-09-27_05-19-56\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-20-14, Step 19/179 in Ep 37, 0.46s  train_loss:1.5500\n",
      "2020-09-27_05-20-30, Step 39/179 in Ep 37, 0.46s  train_loss:1.4284\n",
      "2020-09-27_05-20-48, Step 59/179 in Ep 37, 0.46s  train_loss:1.5330\n",
      "2020-09-27_05-21-08, Step 79/179 in Ep 37, 0.46s  train_loss:1.5361\n",
      "2020-09-27_05-21-34, Step 99/179 in Ep 37, 0.46s  train_loss:1.6438\n",
      "2020-09-27_05-21-59, Step 119/179 in Ep 37, 0.47s  train_loss:1.7871\n",
      "2020-09-27_05-22-22, Step 139/179 in Ep 37, 0.47s  train_loss:1.6944\n",
      "2020-09-27_05-22-50, Step 159/179 in Ep 37, 0.48s  train_loss:1.6129\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-23-08, Step 178/179 in Ep 37, 0.17s  train_loss:12.0008\n",
      "Epoch 37, LR 1.0000000000000005e-07, Train_Time 191.66s, Loss: 1.7491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [01:08<00:00,  1.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8205,  pos_recall: 0.6662 , neg_recall: 0.9749 \n",
      " Acc: 0.8145, Prec: 0.8947, Rec: 0.8745, F1: 0.8831\n",
      "2020-09-27_05-24-17\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-24-36, Step 19/179 in Ep 38, 0.46s  train_loss:1.7126\n",
      "2020-09-27_05-24-59, Step 39/179 in Ep 38, 0.47s  train_loss:1.3733\n",
      "2020-09-27_05-25-20, Step 59/179 in Ep 38, 0.48s  train_loss:1.7261\n",
      "2020-09-27_05-25-40, Step 79/179 in Ep 38, 0.47s  train_loss:1.5226\n",
      "2020-09-27_05-26-01, Step 99/179 in Ep 38, 0.47s  train_loss:1.5169\n",
      "2020-09-27_05-26-20, Step 119/179 in Ep 38, 0.45s  train_loss:1.5967\n",
      "2020-09-27_05-26-40, Step 139/179 in Ep 38, 0.46s  train_loss:1.4082\n",
      "2020-09-27_05-26-57, Step 159/179 in Ep 38, 0.47s  train_loss:1.8916\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-27-14, Step 178/179 in Ep 38, 0.16s  train_loss:8.5196\n",
      "Epoch 38, LR 1.0000000000000005e-07, Train_Time 177.81s, Loss: 1.7210\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:54<00:00,  2.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8188,  pos_recall: 0.6632 , neg_recall: 0.9743 \n",
      " Acc: 0.8152, Prec: 0.8954, Rec: 0.8747, F1: 0.8836\n",
      "2020-09-27_05-28-09\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-28-22, Step 19/179 in Ep 39, 0.46s  train_loss:1.5515\n",
      "2020-09-27_05-28-34, Step 39/179 in Ep 39, 0.45s  train_loss:1.5595\n",
      "2020-09-27_05-28-47, Step 59/179 in Ep 39, 0.47s  train_loss:1.7940\n",
      "2020-09-27_05-29-00, Step 79/179 in Ep 39, 0.46s  train_loss:1.7129\n",
      "2020-09-27_05-29-12, Step 99/179 in Ep 39, 0.46s  train_loss:1.3958\n",
      "2020-09-27_05-29-25, Step 119/179 in Ep 39, 0.46s  train_loss:1.4385\n",
      "2020-09-27_05-29-38, Step 139/179 in Ep 39, 0.46s  train_loss:1.5018\n",
      "2020-09-27_05-29-51, Step 159/179 in Ep 39, 0.47s  train_loss:1.7991\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-30-03, Step 178/179 in Ep 39, 0.16s  train_loss:7.9060\n",
      "Epoch 39, LR 1.0000000000000005e-07, Train_Time 113.91s, Loss: 1.7287\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:34<00:00,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8202,  pos_recall: 0.6659 , neg_recall: 0.9744 \n",
      " Acc: 0.8160, Prec: 0.8956, Rec: 0.8755, F1: 0.8841\n",
      "2020-09-27_05-30-38\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-30-51, Step 19/179 in Ep 40, 0.46s  train_loss:1.9987\n",
      "2020-09-27_05-31-03, Step 39/179 in Ep 40, 0.46s  train_loss:1.6404\n",
      "2020-09-27_05-31-16, Step 59/179 in Ep 40, 0.46s  train_loss:1.9824\n",
      "2020-09-27_05-31-29, Step 79/179 in Ep 40, 0.45s  train_loss:1.4231\n",
      "2020-09-27_05-31-41, Step 99/179 in Ep 40, 0.46s  train_loss:1.9580\n",
      "2020-09-27_05-31-54, Step 119/179 in Ep 40, 0.45s  train_loss:1.6744\n",
      "2020-09-27_05-32-07, Step 139/179 in Ep 40, 0.46s  train_loss:1.7277\n",
      "2020-09-27_05-32-20, Step 159/179 in Ep 40, 0.46s  train_loss:1.8954\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-32-31, Step 178/179 in Ep 40, 0.15s  train_loss:39.9168\n",
      "Epoch 40, LR 1.0000000000000005e-07, Train_Time 113.13s, Loss: 1.8893\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8178,  pos_recall: 0.6619 , neg_recall: 0.9737 \n",
      " Acc: 0.8140, Prec: 0.8939, Rec: 0.8745, F1: 0.8827\n",
      "2020-09-27_05-33-06\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-33-19, Step 19/179 in Ep 41, 0.45s  train_loss:1.3967\n",
      "2020-09-27_05-33-31, Step 39/179 in Ep 41, 0.45s  train_loss:1.6659\n",
      "2020-09-27_05-33-44, Step 59/179 in Ep 41, 0.46s  train_loss:1.9100\n",
      "2020-09-27_05-33-56, Step 79/179 in Ep 41, 0.46s  train_loss:1.5877\n",
      "2020-09-27_05-34-09, Step 99/179 in Ep 41, 0.46s  train_loss:1.7487\n",
      "2020-09-27_05-34-22, Step 119/179 in Ep 41, 0.46s  train_loss:1.6034\n",
      "2020-09-27_05-34-34, Step 139/179 in Ep 41, 0.45s  train_loss:1.6436\n",
      "2020-09-27_05-34-47, Step 159/179 in Ep 41, 0.45s  train_loss:1.4950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-34-58, Step 178/179 in Ep 41, 0.15s  train_loss:11.2916\n",
      "Epoch 41, LR 1.0000000000000005e-07, Train_Time 112.05s, Loss: 1.7507\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8198,  pos_recall: 0.6655 , neg_recall: 0.9742 \n",
      " Acc: 0.8151, Prec: 0.8945, Rec: 0.8755, F1: 0.8835\n",
      "2020-09-27_05-35-34\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-35-47, Step 19/179 in Ep 42, 0.47s  train_loss:1.7001\n",
      "2020-09-27_05-36-00, Step 39/179 in Ep 42, 0.47s  train_loss:1.5388\n",
      "2020-09-27_05-36-13, Step 59/179 in Ep 42, 0.47s  train_loss:1.5623\n",
      "2020-09-27_05-36-25, Step 79/179 in Ep 42, 0.48s  train_loss:2.2097\n",
      "2020-09-27_05-36-39, Step 99/179 in Ep 42, 0.48s  train_loss:1.5585\n",
      "2020-09-27_05-36-53, Step 119/179 in Ep 42, 0.47s  train_loss:1.5789\n",
      "2020-09-27_05-37-05, Step 139/179 in Ep 42, 0.48s  train_loss:1.9347\n",
      "2020-09-27_05-37-19, Step 159/179 in Ep 42, 0.48s  train_loss:1.8907\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-37-30, Step 178/179 in Ep 42, 0.17s  train_loss:13.3200\n",
      "Epoch 42, LR 1.0000000000000005e-08, Train_Time 116.65s, Loss: 1.7629\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:36<00:00,  3.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8192,  pos_recall: 0.6639 , neg_recall: 0.9746 \n",
      " Acc: 0.8153, Prec: 0.8946, Rec: 0.8756, F1: 0.8836\n",
      "2020-09-27_05-38-07\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-38-20, Step 19/179 in Ep 43, 0.48s  train_loss:1.6447\n",
      "2020-09-27_05-38-33, Step 39/179 in Ep 43, 0.48s  train_loss:1.8298\n",
      "2020-09-27_05-38-46, Step 59/179 in Ep 43, 0.48s  train_loss:1.8167\n",
      "2020-09-27_05-39-00, Step 79/179 in Ep 43, 0.47s  train_loss:1.5404\n",
      "2020-09-27_05-39-14, Step 99/179 in Ep 43, 0.47s  train_loss:1.4558\n",
      "2020-09-27_05-39-27, Step 119/179 in Ep 43, 0.47s  train_loss:1.4763\n",
      "2020-09-27_05-39-40, Step 139/179 in Ep 43, 0.48s  train_loss:1.8399\n",
      "2020-09-27_05-39-53, Step 159/179 in Ep 43, 0.47s  train_loss:1.7068\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-40-05, Step 178/179 in Ep 43, 0.17s  train_loss:12.9297\n",
      "Epoch 43, LR 1.0000000000000005e-08, Train_Time 118.02s, Loss: 1.7426\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8178,  pos_recall: 0.6610 , neg_recall: 0.9745 \n",
      " Acc: 0.8151, Prec: 0.8941, Rec: 0.8758, F1: 0.8835\n",
      "2020-09-27_05-40-40\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-40-53, Step 19/179 in Ep 44, 0.45s  train_loss:1.6475\n",
      "2020-09-27_05-41-05, Step 39/179 in Ep 44, 0.46s  train_loss:1.6315\n",
      "2020-09-27_05-41-18, Step 59/179 in Ep 44, 0.46s  train_loss:1.6142\n",
      "2020-09-27_05-41-31, Step 79/179 in Ep 44, 0.45s  train_loss:1.7051\n",
      "2020-09-27_05-41-44, Step 99/179 in Ep 44, 0.46s  train_loss:1.4757\n",
      "2020-09-27_05-41-57, Step 119/179 in Ep 44, 0.47s  train_loss:1.5840\n",
      "2020-09-27_05-42-09, Step 139/179 in Ep 44, 0.46s  train_loss:1.6429\n",
      "2020-09-27_05-42-22, Step 159/179 in Ep 44, 0.48s  train_loss:1.8159\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-42-34, Step 178/179 in Ep 44, 0.17s  train_loss:17.4012\n",
      "Epoch 44, LR 1.0000000000000005e-08, Train_Time 113.99s, Loss: 1.7904\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:36<00:00,  3.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8218,  pos_recall: 0.6688 , neg_recall: 0.9747 \n",
      " Acc: 0.8152, Prec: 0.8939, Rec: 0.8761, F1: 0.8836\n",
      "2020-09-27_05-43-11\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-43-25, Step 19/179 in Ep 45, 0.47s  train_loss:1.8375\n",
      "2020-09-27_05-43-38, Step 39/179 in Ep 45, 0.46s  train_loss:1.5585\n",
      "2020-09-27_05-43-51, Step 59/179 in Ep 45, 0.46s  train_loss:1.6394\n",
      "2020-09-27_05-44-03, Step 79/179 in Ep 45, 0.46s  train_loss:1.6886\n",
      "2020-09-27_05-44-16, Step 99/179 in Ep 45, 0.45s  train_loss:1.8387\n",
      "2020-09-27_05-44-28, Step 119/179 in Ep 45, 0.46s  train_loss:1.5198\n",
      "2020-09-27_05-44-41, Step 139/179 in Ep 45, 0.46s  train_loss:1.6322\n",
      "2020-09-27_05-44-54, Step 159/179 in Ep 45, 0.46s  train_loss:1.4831\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-45-05, Step 178/179 in Ep 45, 0.15s  train_loss:18.4529\n",
      "Epoch 45, LR 1.0000000000000005e-08, Train_Time 112.62s, Loss: 1.7783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8209,  pos_recall: 0.6668 , neg_recall: 0.9750 \n",
      " Acc: 0.8154, Prec: 0.8950, Rec: 0.8754, F1: 0.8837\n",
      "2020-09-27_05-45-41\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-45-53, Step 19/179 in Ep 46, 0.46s  train_loss:1.5629\n",
      "2020-09-27_05-46-06, Step 39/179 in Ep 46, 0.45s  train_loss:1.6596\n",
      "2020-09-27_05-46-19, Step 59/179 in Ep 46, 0.47s  train_loss:1.5298\n",
      "2020-09-27_05-46-32, Step 79/179 in Ep 46, 0.46s  train_loss:1.9115\n",
      "2020-09-27_05-46-44, Step 99/179 in Ep 46, 0.46s  train_loss:1.8480\n",
      "2020-09-27_05-46-57, Step 119/179 in Ep 46, 0.47s  train_loss:2.0026\n",
      "2020-09-27_05-47-10, Step 139/179 in Ep 46, 0.47s  train_loss:1.7249\n",
      "2020-09-27_05-47-22, Step 159/179 in Ep 46, 0.46s  train_loss:2.0613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-47-34, Step 178/179 in Ep 46, 0.17s  train_loss:11.5652\n",
      "Epoch 46, LR 1.0000000000000005e-08, Train_Time 112.67s, Loss: 1.7445\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8192,  pos_recall: 0.6641 , neg_recall: 0.9744 \n",
      " Acc: 0.8148, Prec: 0.8942, Rec: 0.8753, F1: 0.8833\n",
      "2020-09-27_05-48-10\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-48-22, Step 19/179 in Ep 47, 0.45s  train_loss:1.5938\n",
      "2020-09-27_05-48-35, Step 39/179 in Ep 47, 0.47s  train_loss:1.5833\n",
      "2020-09-27_05-48-47, Step 59/179 in Ep 47, 0.46s  train_loss:1.5785\n",
      "2020-09-27_05-49-00, Step 79/179 in Ep 47, 0.46s  train_loss:1.8579\n",
      "2020-09-27_05-49-12, Step 99/179 in Ep 47, 0.46s  train_loss:1.5223\n",
      "2020-09-27_05-49-25, Step 119/179 in Ep 47, 0.47s  train_loss:2.0435\n",
      "2020-09-27_05-49-38, Step 139/179 in Ep 47, 0.46s  train_loss:1.8695\n",
      "2020-09-27_05-49-50, Step 159/179 in Ep 47, 0.46s  train_loss:1.5200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-50-02, Step 178/179 in Ep 47, 0.15s  train_loss:14.9665\n",
      "Epoch 47, LR 1.0000000000000005e-08, Train_Time 112.04s, Loss: 1.7630\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:36<00:00,  3.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8171,  pos_recall: 0.6598 , neg_recall: 0.9744 \n",
      " Acc: 0.8156, Prec: 0.8952, Rec: 0.8756, F1: 0.8839\n",
      "2020-09-27_05-50-38\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-50-51, Step 19/179 in Ep 48, 0.47s  train_loss:1.8544\n",
      "2020-09-27_05-51-03, Step 39/179 in Ep 48, 0.47s  train_loss:1.4652\n",
      "2020-09-27_05-51-16, Step 59/179 in Ep 48, 0.47s  train_loss:1.7760\n",
      "2020-09-27_05-51-29, Step 79/179 in Ep 48, 0.46s  train_loss:1.5520\n",
      "2020-09-27_05-51-41, Step 99/179 in Ep 48, 0.47s  train_loss:1.8078\n",
      "2020-09-27_05-51-54, Step 119/179 in Ep 48, 0.45s  train_loss:1.4965\n",
      "2020-09-27_05-52-06, Step 139/179 in Ep 48, 0.47s  train_loss:1.6844\n",
      "2020-09-27_05-52-19, Step 159/179 in Ep 48, 0.46s  train_loss:1.8115\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-52-30, Step 178/179 in Ep 48, 0.16s  train_loss:15.4525\n",
      "Epoch 48, LR 1.0000000000000005e-08, Train_Time 112.34s, Loss: 1.7791\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8200,  pos_recall: 0.6654 , neg_recall: 0.9747 \n",
      " Acc: 0.8148, Prec: 0.8948, Rec: 0.8748, F1: 0.8833\n",
      "2020-09-27_05-53-06\n",
      "------------------------------------------------------------\n",
      "2020-09-27_05-53-18, Step 19/179 in Ep 49, 0.46s  train_loss:1.6155\n",
      "2020-09-27_05-53-31, Step 39/179 in Ep 49, 0.46s  train_loss:1.8813\n",
      "2020-09-27_05-53-43, Step 59/179 in Ep 49, 0.47s  train_loss:1.8882\n",
      "2020-09-27_05-53-56, Step 79/179 in Ep 49, 0.46s  train_loss:1.7019\n",
      "2020-09-27_05-54-08, Step 99/179 in Ep 49, 0.46s  train_loss:1.8770\n",
      "2020-09-27_05-54-21, Step 119/179 in Ep 49, 0.47s  train_loss:1.9865\n",
      "2020-09-27_05-54-33, Step 139/179 in Ep 49, 0.45s  train_loss:1.5335\n",
      "2020-09-27_05-54-46, Step 159/179 in Ep 49, 0.47s  train_loss:1.6636\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/119 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-27_05-54-58, Step 178/179 in Ep 49, 0.16s  train_loss:10.6084\n",
      "Epoch 49, LR 1.0000000000000005e-08, Train_Time 112.01s, Loss: 1.7165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 119/119 [00:35<00:00,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation on test set, \n",
      " ma: 0.8171,  pos_recall: 0.6601 , neg_recall: 0.9740 \n",
      " Acc: 0.8148, Prec: 0.8946, Rec: 0.8749, F1: 0.8833\n",
      "2020-09-27_05-55-33\n",
      "------------------------------------------------------------\n",
      "PETA,  best_metrc : 0.8217627017103988 in epoch44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\n载入的时候要：\\nfrom tools.function import LogVisual\\nsys.modules['LogVisual'] = LogVisual\\nlog = torch.load('./save/2018-10-29_21:17:34trlog')\\n\""
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#!/usr/bin/env python3\n",
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Sun Jun 14 21:28:44 2020\n",
    "\n",
    "@author: jiahao\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "import os\n",
    "import pprint\n",
    "from collections import OrderedDict, defaultdict\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from batch_engine import valid_trainer, batch_trainer\n",
    "from dataset.AttrDataset import AttrDataset, get_transform\n",
    "from loss.CE_loss import CEL_Sigmoid\n",
    "from models.base_block import FeatClassifier, BaseClassifier\n",
    "from models.resnet import resnet50\n",
    "from models.resnet_hr import resnet50_hr\n",
    "from models.senet import se_resnet101, se_resnet50\n",
    "from models.densenet import densenet121\n",
    "from models.resnext import resnext101_32x4d\n",
    "from models.dpn import dpn68, dpn92\n",
    "from tools.function import get_model_log_path, get_pedestrian_metrics\n",
    "from tools.utils import time_str, save_ckpt, ReDirectSTD, set_seed\n",
    "from datetime import datetime\n",
    "import sys\n",
    "\n",
    "import pandas as pd\n",
    "import torchvision.models as models\n",
    "\n",
    "import argparse\n",
    "\n",
    "# resnet18 = models.resnet18(pretrained=True)\n",
    "\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "set_seed(605)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def argument_parser():\n",
    "    parser = argparse.ArgumentParser(description=\"attribute recognition\",\n",
    "                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n",
    "\n",
    "    parser.add_argument(\"dataset\", type=str, default=\"PETA\")\n",
    "    parser.add_argument(\"--model\", type=str, default=\"se_resnet101\")\n",
    "    parser.add_argument(\"--debug\", action='store_false')\n",
    "\n",
    "    parser.add_argument(\"--batchsize\", type=int, default=64)\n",
    "    parser.add_argument(\"--train_epoch\", type=int, default=50)\n",
    "    parser.add_argument(\"--height\", type=int, default=256)\n",
    "    parser.add_argument(\"--width\", type=int, default=192)\n",
    "    parser.add_argument(\"--lr_ft\", type=float, default=0.01, help='learning rate of feature extractor')\n",
    "    parser.add_argument(\"--lr_new\", type=float, default=0.1, help='learning rate of classifier_base')\n",
    "    parser.add_argument('--classifier', type=str, default='base', help='classifier name')\n",
    "    parser.add_argument('--momentum', type=float, default=0.9)\n",
    "    parser.add_argument('--weight_decay', type=float, default=5e-4)\n",
    "    parser.add_argument(\"--train_split\", type=str, default=\"trainval\", choices=['train', 'trainval'])\n",
    "    parser.add_argument(\"--valid_split\", type=str, default=\"test\", choices=['test', 'valid'])\n",
    "    parser.add_argument('--device', default=0, type=str, help='gpu device ids for CUDA_VISIBLE_DEVICES')\n",
    "    parser.add_argument(\"--redirector\", action='store_false')\n",
    "    parser.add_argument('--use_bn', action='store_false')\n",
    "\n",
    "    return parser\n",
    "\n",
    "def main(args):\n",
    "    visenv_name = args.dataset\n",
    "    exp_dir = os.path.join('exp_result', args.dataset)\n",
    "    model_dir, log_dir = get_model_log_path(exp_dir, visenv_name)\n",
    "    stdout_file = os.path.join(log_dir, f'stdout_{time_str()}.txt')\n",
    "    save_model_path = os.path.join(model_dir, 'ckpt_max.pth')\n",
    "\n",
    "    if args.redirector:\n",
    "        print('redirector stdout')\n",
    "        ReDirectSTD(stdout_file, 'stdout', False)\n",
    "\n",
    "    pprint.pprint(OrderedDict(args.__dict__))\n",
    "\n",
    "    print('-' * 60)\n",
    "    print(f'use GPU{args.device} for training')\n",
    "    print(f'train set: {args.dataset} {args.train_split}, test set: {args.valid_split}')\n",
    "\n",
    "    train_tsfm, valid_tsfm = get_transform(args)\n",
    "    print(train_tsfm)\n",
    "\n",
    "    train_set = AttrDataset(args=args, split=args.train_split, transform=train_tsfm)\n",
    "\n",
    "    train_loader = DataLoader(\n",
    "        dataset=train_set,\n",
    "        batch_size=args.batchsize,\n",
    "        shuffle=True,\n",
    "        num_workers=0,\n",
    "        pin_memory=True,\n",
    "    )\n",
    "    valid_set = AttrDataset(args=args, split=args.valid_split, transform=valid_tsfm)\n",
    "\n",
    "    valid_loader = DataLoader(\n",
    "        dataset=valid_set,\n",
    "        batch_size=args.batchsize,\n",
    "        shuffle=False,\n",
    "        num_workers=0,\n",
    "        pin_memory=True,\n",
    "    )\n",
    "\n",
    "    print(f'{args.train_split} set: {len(train_loader.dataset)}, '\n",
    "          f'{args.valid_split} set: {len(valid_loader.dataset)}, '\n",
    "          f'attr_num : {train_set.attr_num}')\n",
    "\n",
    "    labels = train_set.label\n",
    "    # sample_weight = labels.mean(0)\n",
    "    # sample_weight = labels[labels!=2].reshape((labels.shape[0], labels.shape[1])).mean(0)\n",
    "    sample_weight = np.nanmean(np.where(labels!=2,labels,np.nan), axis=0)\n",
    "\n",
    "    backbone = getattr(sys.modules[__name__], args.model)()\n",
    "    \n",
    "    if \"dpn68\" in args.model:\n",
    "        net_parameter = 832\n",
    "    elif \"dpn\" in args.model:\n",
    "        net_parameter = 2688\n",
    "    elif \"densenet\" in args.model:\n",
    "        net_parameter = 1024\n",
    "    else:\n",
    "        net_parameter = 2048\n",
    "    \n",
    "    classifier = BaseClassifier(netpara=net_parameter, nattr=train_set.attr_num)\n",
    "    model = FeatClassifier(backbone, classifier)\n",
    "\n",
    "    if torch.cuda.is_available():\n",
    "        model = torch.nn.DataParallel(model).cuda()\n",
    "\n",
    "    criterion = CEL_Sigmoid(sample_weight)\n",
    "\n",
    "    if torch.cuda.is_available():\n",
    "        param_groups = [{'params': model.module.finetune_params(), 'lr': args.lr_ft},\n",
    "                        {'params': model.module.fresh_params(), 'lr': args.lr_new}]\n",
    "    else:\n",
    "        param_groups = [{'params': model.finetune_params(), 'lr': args.lr_ft},\n",
    "                        {'params': model.fresh_params(), 'lr': args.lr_new}]\n",
    "    optimizer = torch.optim.SGD(param_groups, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=False)\n",
    "    lr_scheduler = ReduceLROnPlateau(optimizer, factor=0.1, patience=4)\n",
    "\n",
    "    best_metric, epoch = trainer(epoch=args.train_epoch,\n",
    "                                 model=model,\n",
    "                                 train_loader=train_loader,\n",
    "                                 valid_loader=valid_loader,\n",
    "                                 criterion=criterion,\n",
    "                                 optimizer=optimizer,\n",
    "                                 lr_scheduler=lr_scheduler,\n",
    "                                 path=save_model_path)\n",
    "\n",
    "    print(f'{visenv_name},  best_metrc : {best_metric} in epoch{epoch}')\n",
    "\n",
    "\n",
    "def trainer(epoch, model, train_loader, valid_loader, criterion, optimizer, lr_scheduler,\n",
    "            path):\n",
    "    maximum = float(-np.inf)\n",
    "    best_epoch = 0\n",
    "\n",
    "    result_list = defaultdict()\n",
    "    \n",
    "    df_metrics = pd.DataFrame(columns=['epoch', 'train_loss', 'train_instance_acc', 'train_instance_prec', 'train_instance_recall', 'train_instance_f1', 'train_ma', 'train_pos_recall', 'train_neg_recall',\n",
    "                                   'valid_loss', 'valid_instance_acc', 'valid_instance_prec', 'valid_instance_recall', 'valid_instance_f1', 'valid_ma', 'valid_pos_recall', 'valid_neg_recall'])\n",
    "\n",
    "\n",
    "    for i in range(epoch):\n",
    "\n",
    "        train_loss, train_gt, train_probs = batch_trainer(\n",
    "            epoch=i,\n",
    "            model=model,\n",
    "            train_loader=train_loader,\n",
    "            criterion=criterion,\n",
    "            optimizer=optimizer,\n",
    "        )\n",
    "\n",
    "        valid_loss, valid_gt, valid_probs = valid_trainer(\n",
    "            model=model,\n",
    "            valid_loader=valid_loader,\n",
    "            criterion=criterion,\n",
    "        )\n",
    "\n",
    "        lr_scheduler.step(metrics=valid_loss, epoch=i)\n",
    "\n",
    "        train_result = get_pedestrian_metrics(train_gt, train_probs)\n",
    "        valid_result = get_pedestrian_metrics(valid_gt, valid_probs)\n",
    "\n",
    "        # tensorboard added\n",
    "        # writer.add_scalar(tag, function, iteration)\n",
    "        writer_step = i\n",
    "\n",
    "        writer.add_scalars('Loss', {'Train':train_loss, 'Valid':valid_loss}, writer_step)\n",
    "        writer.add_scalars('Accuracy', {'Train':train_result.instance_acc, 'Valid':valid_result.instance_acc}, writer_step)\n",
    "        writer.add_scalars('Precision', {'Train':train_result.instance_prec, 'Valid':valid_result.instance_prec}, writer_step)\n",
    "        writer.add_scalars('Recall', {'Train':train_result.instance_recall, 'Valid':valid_result.instance_recall}, writer_step)\n",
    "        writer.add_scalars('F1', {'Train':train_result.instance_f1, 'Valid':valid_result.instance_f1}, writer_step)\n",
    "        writer.add_scalars('Mean Accuracy', {'Train':train_result.ma, 'Valid':valid_result.ma}, writer_step)\n",
    "        writer.add_scalars('Pos Recall', {'Train':np.mean(train_result.label_pos_recall), 'Valid':np.mean(valid_result.label_pos_recall)}, writer_step)\n",
    "        writer.add_scalars('Neg Recall', {'Train':np.mean(train_result.label_neg_recall), 'Valid':np.mean(valid_result.label_neg_recall)}, writer_step)\n",
    "\n",
    "        print(f'Evaluation on test set, \\n',\n",
    "              'ma: {:.4f},  pos_recall: {:.4f} , neg_recall: {:.4f} \\n'.format(\n",
    "                  valid_result.ma, np.mean(valid_result.label_pos_recall), np.mean(valid_result.label_neg_recall)),\n",
    "              'Acc: {:.4f}, Prec: {:.4f}, Rec: {:.4f}, F1: {:.4f}'.format(\n",
    "                  valid_result.instance_acc, valid_result.instance_prec, valid_result.instance_recall,\n",
    "                  valid_result.instance_f1))\n",
    "\n",
    "        print(f'{time_str()}')\n",
    "        print('-' * 60)\n",
    "        \n",
    "        # create metrics dataframe to save as csv\n",
    "\n",
    "        new_metrics = { \n",
    "            'epoch':i,\n",
    "            'train_loss':train_loss,\n",
    "            'train_instance_acc':train_result.instance_acc,\n",
    "            'train_instance_prec':train_result.instance_prec,\n",
    "            'train_instance_recall':train_result.instance_recall,\n",
    "            'train_instance_f1':train_result.instance_f1,\n",
    "            'train_ma':train_result.ma,\n",
    "            'train_pos_recall':np.mean(train_result.label_pos_recall),\n",
    "            'train_neg_recall':np.mean(train_result.label_neg_recall),\n",
    "            'valid_loss':valid_loss,\n",
    "            'valid_instance_acc':valid_result.instance_acc,\n",
    "            'valid_instance_prec':valid_result.instance_prec,\n",
    "            'valid_instance_recall':valid_result.instance_recall,\n",
    "            'valid_instance_f1':valid_result.instance_f1,\n",
    "            'valid_ma':valid_result.ma,\n",
    "            'valid_pos_recall':np.mean(valid_result.label_pos_recall),\n",
    "            'valid_neg_recall':np.mean(valid_result.label_neg_recall)\n",
    "            }\n",
    "        #append row to the dataframe\n",
    "        df_metrics = df_metrics.append(new_metrics, ignore_index=True)\n",
    "        df_metrics.to_csv(csv_file_name, index=False)\n",
    "\n",
    "        cur_metric = valid_result.ma\n",
    "\n",
    "        if cur_metric > maximum:\n",
    "            maximum = cur_metric\n",
    "            best_epoch = i\n",
    "            save_ckpt(model, path, i, maximum)\n",
    "\n",
    "        result_list[i] = [train_result, valid_result]\n",
    "\n",
    "    writer.close()\n",
    "\n",
    "    torch.save(result_list, os.path.join(os.path.dirname(path), 'metric_log.pkl'))\n",
    "\n",
    "    return maximum, best_epoch\n",
    "\n",
    "\n",
    "parser = argument_parser()\n",
    "args = parser.parse_args([\"PETA\"])\n",
    "\n",
    "log_dir = 'runs/' + args.dataset+\"_\"+args.model+\"_\"+datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "csv_dir = 'csv_folder/'\n",
    "if not os.path.exists(csv_dir):\n",
    "    os.makedirs(csv_dir)\n",
    "csv_file_name = os.path.join(csv_dir, args.dataset+\"_\"+args.model+\"_\"+datetime.now().strftime(\"%Y%m%d-%H%M%S\") +'.csv')\n",
    "writer = SummaryWriter(log_dir)\n",
    "\n",
    "main(args)\n",
    "\n",
    "\n",
    "    # os.path.abspath()\n",
    "\n",
    "\"\"\"\n",
    "载入的时候要：\n",
    "from tools.function import LogVisual\n",
    "sys.modules['LogVisual'] = LogVisual\n",
    "log = torch.load('./save/2018-10-29_21:17:34trlog')\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python3\n",
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Sat Sep  5 21:45:27 2020\n",
    "\n",
    "@author: jiahao\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "import pickle\n",
    "import random\n",
    "import shutil\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from easydict import EasyDict\n",
    "from scipy.io import loadmat\n",
    "\n",
    "np.random.seed(0)\n",
    "random.seed(0)\n",
    "\n",
    "with open('/hpctmp/e0384293/Capstone_without_data/data/PETA+RAP/dataset.pkl', 'rb') as f:\n",
    "    peta_pkl = pickle.load(f)\n",
    "    \n",
    "peta_pkl.root = '/hpctmp/e0384293/Capstone_without_data/data/PETA+RAP/images'\n",
    "\n",
    "with open('/hpctmp/e0384293/Capstone_without_data/data/PETA+RAP/dataset_new.pkl', 'wb') as f:\n",
    "    pickle.dump(peta_pkl, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
